/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package uit.tkorg.coauthorgraph.servlet.crawl;

import java.io.IOException;
import java.io.PrintWriter;
import java.net.URL;
import java.util.ArrayList;
import java.util.logging.FileHandler;
import java.util.logging.Handler;
import java.util.logging.Level;
import java.util.logging.LogRecord;
import java.util.logging.Logger;
import javax.servlet.RequestDispatcher;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import uit.tkorg.coauthorgraph.bo.AuthorBO;
import uit.tkorg.coauthorgraph.bo.AuthorPaperBO;
import uit.tkorg.coauthorgraph.bo.ConferenceBO;
import uit.tkorg.coauthorgraph.bo.JournalBO;
import uit.tkorg.coauthorgraph.bo.PaperBO;
import uit.tkorg.coauthorgraph.dto.AuthorDTO;
import uit.tkorg.coauthorgraph.dto.ConferenceDTO;
import uit.tkorg.coauthorgraph.dto.CrawlAuthorDTO;
import uit.tkorg.coauthorgraph.dto.JournalDTO;
import uit.tkorg.coauthorgraph.dto.PaperDTO;
import uit.tkorg.coauthorgraph.property.file.MASearchCrawlPatterns;
import uit.tkorg.coauthorgraph.utility.CommonHTMLProcessor;
import uit.tkorg.coauthorgraph.utility.CommonURLQuery;
import uit.tkorg.coauthorgraph.utility.MASFetcher;
import uit.tkorg.coauthorgraph.utility.MASResultAnalyze;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.logging.Formatter;

/**
 *
 * @author Huynh Ngoc Tin
 */
public class MASAutoCrawlingServlet extends HttpServlet {

    public static Logger logger;
    public static Handler logHandler;

    static {
        try {
            boolean append = true;
            logHandler = new FileHandler("C:\\CrawlerLog\\Crawler.log", 10000000, 1000, append);
            logHandler.setLevel(Level.ALL);
            logHandler.setFormatter(new Formatter() {

                public String format(LogRecord logRecord) {
                    StringBuffer strBuffer = new StringBuffer(10000);
                    strBuffer.append("[ ");
                    strBuffer.append(new java.util.Date());
                    strBuffer.append(" ] - ");
                    strBuffer.append(logRecord.getLevel());
                    strBuffer.append(": ");
                    strBuffer.append(formatMessage(logRecord));
                    strBuffer.append('\n');
                    return strBuffer.toString();
                }
            });

            logger = Logger.getLogger("CrawlerLogger");
            logger.setUseParentHandlers(false);
            logger.addHandler(logHandler);
            // Change LOGGING_LEVEL in Constant to WARNING in order to disable logging on normal info, only log important info.
            logger.setLevel(Level.INFO);
        } catch (Exception ex) {
            ex.printStackTrace();
        }
    }

    protected void service(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
        StringBuffer bufferLog = new StringBuffer();

        request.setCharacterEncoding("UTF-8");
        PrintWriter out = response.getWriter();
        String[] authorList = request.getParameterValues("authorList");
        String depthLevel = request.getParameter("depthLevel");

        ArrayList notCrawlAuthorList = new ArrayList();
        ArrayList alreadyCrawlAuthorList = new ArrayList();
        CrawlAuthorDTO crawlingAuthorDTO = null;
        if (authorList != null) {
            for (int i = 0; i < authorList.length; i++) {
                crawlingAuthorDTO = new CrawlAuthorDTO();
                crawlingAuthorDTO.setAuthorName(authorList[i]);
                crawlingAuthorDTO.setCrawlDepthLevel(1);
                notCrawlAuthorList.add(crawlingAuthorDTO);

                // Begin of Crawling & Importing Progress
                logger.info("************************************************************");
                logger.info("STARTING NUMBER OF AUTHORS: " + authorList.length);
                logger.info("************************************************************");
                logger.info("************************************************************");
                logger.info("START CRAWLING AUTHOR NO." + (i+1) + " IN THE LIST");
                logger.info("************************************************************");

                while ((notCrawlAuthorList != null) && (notCrawlAuthorList.size() > 0)) {
                    crawlingAuthorDTO = (CrawlAuthorDTO) notCrawlAuthorList.get(0);
                    // Check if the depth level is reached and stop crawling & importing progress.
                    if (!depthLevel.equalsIgnoreCase("Unlimited")) {
                        int curCrawlingDepthLevel = crawlingAuthorDTO.getCrawlDepthLevel();
                        if (curCrawlingDepthLevel > Integer.parseInt(depthLevel)) {
                            notCrawlAuthorList.clear();
                            break;
                        }
                    }

                    logger.info("************************************************************");
                    logger.info("*** Crawling for author:" + crawlingAuthorDTO.getAuthorName());
                    logger.info("*** Current Depth Level:" + crawlingAuthorDTO.getCrawlDepthLevel());
                    logger.info("************************************************************");

                    autoCrawlAndImportForAnAuthor(crawlingAuthorDTO.getAuthorName(), depthLevel, request, response);
                    alreadyCrawlAuthorList.add(crawlingAuthorDTO);

                    // get the co-cothor list of the crawling author to add to the list 'notCrawlAuthorList'
                    try {
                        AuthorPaperBO authorPaperBO = new AuthorPaperBO();
                        ArrayList coAuthorList = authorPaperBO.getCoAuthorList(crawlingAuthorDTO.getAuthorName());
                        if (coAuthorList != null) {
                            for (int j = 0; j < coAuthorList.size(); j++) {
                                AuthorDTO curAuthorDTO = (AuthorDTO) coAuthorList.get(j);
                                String curCoAuthorName = curAuthorDTO.getAuthorName();
                                int curCoAuthorID = curAuthorDTO.getAuthorID();
                                // Check if this author exist in the 'notCrawlAuthorList' or not
                                boolean found = false;
                                for (int k = 0; k < notCrawlAuthorList.size(); k++) {
                                    CrawlAuthorDTO notCrawlAuthorDTO = (CrawlAuthorDTO) notCrawlAuthorList.get(k);
                                    if (curCoAuthorName.equalsIgnoreCase(notCrawlAuthorDTO.getAuthorName())) {
                                        found = true;
                                        break;
                                    }
                                }
                                
                                // Check if this author exist in the 'alreadyCrawlAuthorList' or not
                                for (int m = 0; m < alreadyCrawlAuthorList.size(); m++) {
                                    CrawlAuthorDTO alreadyCrawlAuthorDTO = (CrawlAuthorDTO) alreadyCrawlAuthorList.get(m);
                                    if (curCoAuthorName.equalsIgnoreCase(alreadyCrawlAuthorDTO.getAuthorName())) {
                                        found = true;
                                        break;
                                    }
                                }
                                
                                // add a new co-author into the 'notCrawlAuthorList' list.
                                if (found == false) {
                                    CrawlAuthorDTO newCrawlAuthorDTO = new CrawlAuthorDTO();
                                    newCrawlAuthorDTO.setAuthorName(curCoAuthorName);
                                    newCrawlAuthorDTO.setCrawlDepthLevel(crawlingAuthorDTO.getCrawlDepthLevel() + 1);
                                    notCrawlAuthorList.add(newCrawlAuthorDTO);
                                }
                            }
                        }

                        // remove the first CrawlAuthorDTO in the 'notCrawlAuthorList' list
                        notCrawlAuthorList.remove(0);
                    } catch (Exception ex) {
                        ex.printStackTrace();
                        logger.warning(ex.toString());
                    }
                } // end of while(...) - End of of Crawling & Importing Progress

                // clear all list
                notCrawlAuthorList.clear();
                alreadyCrawlAuthorList.clear();
            }
        }

        logger.info("************************************************************");
        logger.info("-------------------DONE DONE DONE --------------------------");
        logger.info("************************************************************");
        String resultMessage = "Please take a look at the Console to see the crawl & import progress";
        request.setAttribute("resultMessage", resultMessage);
        RequestDispatcher dispatcher = request.getRequestDispatcher("./jsp/MASAutoCrawling.jsp");
        dispatcher.forward(request, response);
    }

    /**
     * Crawl publications & other information relate to an author from MAS
     * @param authorName
     * @param depthLevel
     * @param request
     */
    public void autoCrawlAndImportForAnAuthor(String crawlAuthorName, String depthLevel, HttpServletRequest request, HttpServletResponse response) {
        CommonHTMLProcessor htmlParser = new CommonHTMLProcessor();
        MASFetcher fetcher = new MASFetcher();
        String _pageContent = fetcher.fetchHTMLContentForAuthor(crawlAuthorName);
        int checkStatus = 0;;
        // Collect all publications at returned pages
        try {
            // get number of publications of this author in DB
            AuthorPaperBO authorPaperBO = new AuthorPaperBO();
            double totalPaperOfAuthor = authorPaperBO.countTotalPaperOfAuthor(crawlAuthorName);
            ArrayList publicationList = MASResultAnalyze.getPublications(_pageContent, crawlAuthorName, totalPaperOfAuthor);
            if (publicationList != null && publicationList.size() > 0) {
                importMetaDataPublicationForAllCrawledPapers(publicationList);
            } else {
                System.out.print("THE AUTHOR EXISTED AND HAVE NO ANY NEW PUBLICATIONS OR NOTTHING TO CRAWL");
            }
        } catch (Exception ex) {
            ex.printStackTrace();
            logger.warning(ex.toString());
        }
    }

    /**
     * import a specified list of publications into the DB
     * @param publicationList
     */
    private void importMetaDataPublicationForAllCrawledPapers(ArrayList publicationList) {
        try {
            PaperBO paperBO = new PaperBO();
            AuthorBO authorBO = new AuthorBO();

            PaperDTO paperDTO;
            // Import the metadata information for all papers that crawled from MAS
            System.out.println("*** Number of publications:" + publicationList.size());
            for (int i = 0; i < publicationList.size(); i++) {
                paperDTO = (PaperDTO) publicationList.get(i);
                if (!paperBO.isPaperExisted(paperDTO)) { // LOI O DAY
                    System.out.println((i + 1) + "." + paperDTO.getTitle() + ".");

                    // insert conference of this paper
                    String confName = paperDTO.getConferenceName();
                    if (confName != null && !confName.equalsIgnoreCase("")) {
                        ConferenceBO confBO = new ConferenceBO();
                        ConferenceDTO confDTO = new ConferenceDTO();
                        confDTO.setConferenceName(confName);
                        confDTO.setYear(paperDTO.getYear());
                        int confID = confBO.getConferenceID(confName, paperDTO.getYear());
                        if (confID == -1) {
                            int insertConfID = confBO.insertNewConference(confDTO);
                            paperDTO.setConferenceID(insertConfID);
                        } else {
                            // this paper relate to a conference that existed inDB
                            paperDTO.setConferenceID(confID);
                        }
                    }

                    // insert journal of this paper
                    String journalName = paperDTO.getJournalName();
                    if (journalName != null && !journalName.equalsIgnoreCase("")) {
                        JournalBO journalBO = new JournalBO();
                        JournalDTO journalDTO = new JournalDTO();
                        journalDTO.setJournalName(journalName);
                        journalDTO.setYear(paperDTO.getYear());
                        int journalID = journalBO.getJournalID(journalName, paperDTO.getYear());
                        if (journalID == -1) {
                            int insertJourID = journalBO.insertNewJournal(journalDTO);
                            paperDTO.setJournalID(insertJourID);
                        } else {
                            // this paper relate to a journal that existed inDB
                            paperDTO.setJournalID(journalID);
                        }
                    }

                    // insert a new paper into the 'paper' table
                    int insertPaperID = paperBO.insertNewPaper(paperDTO);

                    // insert authors into the tables ('author', 'authorinstance', 'authorpaper')
                    ArrayList authors = paperDTO.getAuthorNameList();
                    for (int j = 0; j < authors.size(); j++) {
                        String importAuthorName = (String) authors.get(j);
                        importAuthorName = importAuthorName.replaceAll("(\r\n|\r|\n|\n\r)", "");
                        int importAuthorID = authorBO.getAuthorID(importAuthorName);
                        if (importAuthorID == -1) { // not exist in the 'Author' Table
                            // update information for authorDTO object
                            AuthorDTO authorDTO = new AuthorDTO();
                            authorDTO.setAuthorName(importAuthorName);
                            importAuthorID = authorBO.insertNewAuthor(authorDTO);
                        }

                        AuthorPaperBO author_paper_BO = new AuthorPaperBO();
                        if (!author_paper_BO.isAuthorPaperExist(importAuthorID, insertPaperID)) {
                            // insert authorID, paperID into the 'AuthorPaper' table
                            int author_OrderNum = j + 1;
                            author_paper_BO.insertNewAuthorPaper(importAuthorID, insertPaperID, author_OrderNum);
                        }
                    }
                } else { // the paper is existed
                    System.out.println((i + 1) + "." + paperDTO.getTitle() + ". " + "EXISTED, NOT INSERT!");
                }
            } // end of for (...)
        } catch (Exception ex) {
            ex.printStackTrace();
            logger.warning(ex.toString());
        }
    }
}
